<!doctype html>



  


<html class="theme-next mist use-motion">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>



<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />












  <link href="/vendors/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css"/>




  <link href="//fonts.googleapis.com/css?family=Lato:300,400,700,400italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">



<link href="/vendors/font-awesome/css/font-awesome.min.css?v=4.4.0" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=0.5.0" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="Machine Learning, Data Mining, Deep Learning" />





  <link rel="alternate" href="/atom.xml" title="Snakecy's NOTE" type="application/atom+xml" />




  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=0.5.0" />






<meta name="description" content="The unexamined life is not worth living. --Socrates">
<meta property="og:type" content="website">
<meta property="og:title" content="Snakecy's NOTE">
<meta property="og:url" content="https://snakecy.github.io/page/4/index.html">
<meta property="og:site_name" content="Snakecy's NOTE">
<meta property="og:description" content="The unexamined life is not worth living. --Socrates">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Snakecy's NOTE">
<meta name="twitter:description" content="The unexamined life is not worth living. --Socrates">



<script type="text/javascript" id="hexo.configuration">
  var NexT = window.NexT || {};
  var CONFIG = {
    scheme: 'Mist',
    sidebar: {"position":"right","display":"post"},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: undefined,
      author: 'Author'
    }
  };
</script>

  <title> Snakecy's NOTE </title>
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="en">

  


<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
            (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
          m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
  ga('create', '[object Object]', 'auto');
  ga('send', 'pageview');
</script>




  <script type="text/javascript">
    (function() {
      var hm = document.createElement("script");
      hm.src = "//tajs.qq.com/stats?sId=55430358";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>




  
  
    
  

  <div class="container one-collumn sidebar-position-right 
   page-home 
 ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-meta ">
  

  <div class="custom-logo-site-title">
    <a href="/"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">Snakecy's NOTE</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle"></p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-home fa-fw"></i> <br />
            
            Home
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives" rel="section">
            
              <i class="menu-item-icon fa fa-archive fa-fw"></i> <br />
            
            Archives
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about" rel="section">
            
              <i class="menu-item-icon fa fa-user fa-fw"></i> <br />
            
            About
          </a>
        </li>
      

      
        <li class="menu-item menu-item-search">
          
            <a href="#" class="st-search-show-outputs">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br />
            
            Search
          </a>
        </li>
      
    </ul>
  

  
    <div class="site-search">
      
  <form class="site-search-form">
  <input type="text" id="st-search-input" class="st-search-input st-default-search-input" />
</form>

<script type="text/javascript">
  (function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
    (w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t);
    e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e);
  })(window,document,'script','//s.swiftypecdn.com/install/v2/st.js','_st');

  _st('install', 'Gp7n4GTCVBzWyRCst3ez','2.0.0');
</script>



    </div>
  
</nav>

 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            
  <section id="posts" class="posts-expand">
    
      

  
  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2016/01/15/UCI-machine-learning-repository/" itemprop="url">
                  UCI Machine Learning Repository
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            <span class="post-meta-item-icon">
              <i class="fa fa-calendar-o"></i>
            </span>
            <span class="post-meta-item-text">Posted on</span>
            <time itemprop="dateCreated" datetime="2016-01-15T16:46:41+08:00" content="2016-01-15">
              2016-01-15
            </time>
          </span>

          
            <span class="post-category" >
              &nbsp; | &nbsp;
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">In</span>
              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/algorithm/" itemprop="url" rel="index">
                    <span itemprop="name">algorithm</span>
                  </a>
                </span>

                
                

              
            </span>
          

          
            
              <span class="post-comments-count">
                &nbsp; | &nbsp;
                <a href="/2016/01/15/UCI-machine-learning-repository/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count disqus-comment-count" data-disqus-identifier="2016/01/15/UCI-machine-learning-repository/" itemprop="commentsCount"></span>
                </a>
              </span>
            
          

          

          
          
             <span id="/2016/01/15/UCI-machine-learning-repository/" class="leancloud_visitors" data-flag-title="UCI Machine Learning Repository">
               &nbsp; | &nbsp;
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               <span class="post-meta-item-text">Views </span>
               <span class="leancloud-visitors-count"></span>
              </span>
          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <h2 id="Datasets_Examples_for_machine_learning"><a href="#Datasets_Examples_for_machine_learning" class="headerlink" title="Datasets Examples for machine learning"></a>Datasets Examples for machine learning</h2><p>Here are the data sets from <a href="http://archive.ics.uci.edu/ml/" target="_blank" rel="external">UCI Machine Learning Repository</a> once I sorted and practiced. Some of the abstracts I summarized updated to my github <a href="https://github.com/snakecy/CUIMachineLearningRepositry" target="_blank" rel="external">CUIMachineLearningRepository</a>.</p>
<p>Example:</p>
<h3 id="Image_Segmentation"><a href="#Image_Segmentation" class="headerlink" title="Image Segmentation"></a>Image Segmentation</h3><ol>
<li>Data set website<br><a href="http://archive.ics.uci.edu/ml/datasets/Image+Segmentation" target="_blank" rel="external">http://archive.ics.uci.edu/ml/datasets/Image+Segmentation</a></li>
<li>Datasets describe<br>【1】The data used in our experiments were collected by Vision Group, University of Massachusetts.【2】We used to classify the image.【3】The instances were drawn randomly from a database of 7 outdoor images. The images were hand segmented to create a classification for every pixel. Each instance is a 3x3 region. Number of Attributes has 19 continuous attributes, each attribute can describes as region-centroid-col、region-centroid-row、region-pixel-count、short-line-density-5、short-line-density-2、 vedge-mean、vegde-sd、hedge-mean、hedge-sd、intensity-mean、rawred-mean、rawblue-mean、rawgreen-mean、exred-mean、exblue-mean、exgreen-mean、value-mean、saturatoin-mean、hue-mean.【4】The database has 2310 samples, respectively belong to training with 210 samples and testing with 2100 samples. The categories of network system include seven categories, as shown in Table 1.</li>
</ol>
<h5 id="Table_1_Category_Distribution_of_Network_System"><a href="#Table_1_Category_Distribution_of_Network_System" class="headerlink" title="Table 1 Category Distribution of Network System"></a>Table 1 Category Distribution of Network System</h5><table>
<thead>
<tr>
<th style="text-align:left">Invasion</th>
<th style="text-align:left">Training</th>
<th style="text-align:left">Testing</th>
<th style="text-align:left">Total Number of Samples</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:left">Brickface</td>
<td style="text-align:left">30</td>
<td style="text-align:left">300</td>
<td style="text-align:left">330</td>
</tr>
<tr>
<td style="text-align:left">sky</td>
<td style="text-align:left">30</td>
<td style="text-align:left">300</td>
<td style="text-align:left">330</td>
</tr>
<tr>
<td style="text-align:left">Foliage</td>
<td style="text-align:left">30</td>
<td style="text-align:left">300</td>
<td style="text-align:left">330</td>
</tr>
<tr>
<td style="text-align:left">Cement</td>
<td style="text-align:left">30</td>
<td style="text-align:left">300</td>
<td style="text-align:left">330</td>
</tr>
<tr>
<td style="text-align:left">window</td>
<td style="text-align:left">30</td>
<td style="text-align:left">300</td>
<td style="text-align:left">330</td>
</tr>
<tr>
<td style="text-align:left">Path</td>
<td style="text-align:left">30</td>
<td style="text-align:left">300</td>
<td style="text-align:left">330</td>
</tr>
<tr>
<td style="text-align:left">grass</td>
<td style="text-align:left">30</td>
<td style="text-align:left">300</td>
<td style="text-align:left">330</td>
</tr>
<tr>
<td style="text-align:left">Total number of samples in total</td>
<td style="text-align:left">210</td>
<td style="text-align:left">2100</td>
<td style="text-align:left">2310</td>
</tr>
</tbody>
</table>

          
        
      
    </div>

    <div>
      
    </div>

    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  
  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2016/01/13/nosql-database/" itemprop="url">
                  Install NoSQL Database
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            <span class="post-meta-item-icon">
              <i class="fa fa-calendar-o"></i>
            </span>
            <span class="post-meta-item-text">Posted on</span>
            <time itemprop="dateCreated" datetime="2016-01-13T02:25:57+08:00" content="2016-01-13">
              2016-01-13
            </time>
          </span>

          
            <span class="post-category" >
              &nbsp; | &nbsp;
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">In</span>
              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/open-source/" itemprop="url" rel="index">
                    <span itemprop="name">open-source</span>
                  </a>
                </span>

                
                

              
            </span>
          

          
            
              <span class="post-comments-count">
                &nbsp; | &nbsp;
                <a href="/2016/01/13/nosql-database/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count disqus-comment-count" data-disqus-identifier="2016/01/13/nosql-database/" itemprop="commentsCount"></span>
                </a>
              </span>
            
          

          

          
          
             <span id="/2016/01/13/nosql-database/" class="leancloud_visitors" data-flag-title="Install NoSQL Database">
               &nbsp; | &nbsp;
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               <span class="post-meta-item-text">Views </span>
               <span class="leancloud-visitors-count"></span>
              </span>
          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>NoSQL database for a key-value DB Redis</p>
<ul>
<li><p>Quick Start <a href="http://redis.io/topics/quickstart" target="_blank" rel="external">Redis</a></p>
</li>
<li><p>use the key value DB (redis)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get install redis-server</span><br><span class="line">to <span class="built_in">test</span> redis using: &gt;&gt; redis-cli   to enter</span><br><span class="line"><span class="keyword">then</span> : 127.0.0.1:6379&gt; <span class="built_in">set</span> <span class="built_in">test</span> 1</span><br><span class="line">&gt; get <span class="built_in">test</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>Or download the redis from website</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">wget http://download.redis.io/releases/redis-3.0.2.tar.gz</span><br></pre></td></tr></table></figure>
</li>
<li><p>Or download</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">wget http://download.redis.io/redis-stable.tar.gz</span><br><span class="line">tar xvzf redis-stable.tar.gz</span><br><span class="line"><span class="built_in">cd</span> redis-stable</span><br><span class="line">make</span><br></pre></td></tr></table></figure>
</li>
<li><p>Update the redis-server</p>
<ul>
<li><p>first way</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">sudo add-apt-repository ppa:chris-lea/redis-server</span><br><span class="line">sudo apt-get update</span><br><span class="line">sudo apt-get install redis-server</span><br></pre></td></tr></table></figure>
</li>
<li><p>another way to do this update</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get install -y python-software-properties</span><br><span class="line">sudo add-apt-repository -y ppa:rwky/redis</span><br><span class="line">sudo apt-get update</span><br><span class="line">sudo apt-get install -y redis-server</span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
</ul>

          
        
      
    </div>

    <div>
      
    </div>

    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  
  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2016/01/13/json-eclipse/" itemprop="url">
                  Json parsered by Java
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            <span class="post-meta-item-icon">
              <i class="fa fa-calendar-o"></i>
            </span>
            <span class="post-meta-item-text">Posted on</span>
            <time itemprop="dateCreated" datetime="2016-01-13T01:26:05+08:00" content="2016-01-13">
              2016-01-13
            </time>
          </span>

          
            <span class="post-category" >
              &nbsp; | &nbsp;
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">In</span>
              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/open-source/" itemprop="url" rel="index">
                    <span itemprop="name">open-source</span>
                  </a>
                </span>

                
                

              
            </span>
          

          
            
              <span class="post-comments-count">
                &nbsp; | &nbsp;
                <a href="/2016/01/13/json-eclipse/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count disqus-comment-count" data-disqus-identifier="2016/01/13/json-eclipse/" itemprop="commentsCount"></span>
                </a>
              </span>
            
          

          

          
          
             <span id="/2016/01/13/json-eclipse/" class="leancloud_visitors" data-flag-title="Json parsered by Java">
               &nbsp; | &nbsp;
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               <span class="post-meta-item-text">Views </span>
               <span class="leancloud-visitors-count"></span>
              </span>
          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>Describe how to parser Json flows in (eclipse)  Java, also in Scala</p>
<h2 id="Json_in_Java"><a href="#Json_in_Java" class="headerlink" title="Json in Java"></a>Json in Java</h2><p>Json-lib must contain the jars blowing ( versions not limited)</p>
<ul>
<li><p><a href="http://www.studytrails.com/java/json/java-org-json.jsp" target="_blank" rel="external">Java-org-json</a></p>
</li>
<li><p>Json-lib</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">commons-beanutils-1.7.0.jar  or commons-beanutils-1.9.2.jar</span><br><span class="line">commons-collections-3.1.jar  or commons-collections-3.2.1.jar</span><br><span class="line">commons-lang-2.5.jar  or commons-lang-2.6.jar</span><br><span class="line">commons-logging-1.1.1.jar  or commons-logging-1.2.jar</span><br><span class="line">ezmorph-1.0.3.jar  or ezmorph-1.0.6.jar</span><br><span class="line">json-lib-2.2.2-jdk15.jar  or json-lib-2.4-jdk15.jar</span><br></pre></td></tr></table></figure>
</li>
<li><p><a href="http://www.cnblogs.com/zhenjing/p/json-smart.html" target="_blank" rel="external">Json-smart</a></p>
<ul>
<li>Compare with each json jar package <a href="https://code.google.com/p/json-smart/wiki/FeaturesTests" target="_blank" rel="external">compare</a></li>
</ul>
</li>
<li><p>simple-json</p>
</li>
<li><p>org.json</p>
</li>
<li><p>example</p>
<ul>
<li><a href="http://crunchify.com/how-to-write-json-object-to-file-in-java/" target="_blank" rel="external">http://crunchify.com/how-to-write-json-object-to-file-in-java/</a></li>
<li><a href="http://crunchify.com/how-to-read-json-object-from-file-in-java/" target="_blank" rel="external">http://crunchify.com/how-to-read-json-object-from-file-in-java/</a></li>
</ul>
</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># reference:</span></span><br><span class="line">&gt; json格式如下：&#123;<span class="string">"response"</span>:&#123;<span class="string">"data"</span>:[&#123;<span class="string">"address"</span>:<span class="string">"南京市游乐园"</span>,<span class="string">"province"</span>:<span class="string">"江苏"</span>,<span class="string">"district"</span>:<span class="string">"玄武区"</span>,<span class="string">"city"</span>:<span class="string">"南京"</span>&#125;]&#125;,<span class="string">"status"</span>:<span class="string">"ok"</span>&#125;</span><br><span class="line">Result： 江苏 南京 玄武区 南京市游乐园</span><br><span class="line">﻿</span><br><span class="line">JSONObject dataJson=new JSONObject(<span class="string">"Json data"</span>);</span><br><span class="line">JSONObject response=dataJson.getJSONObject(<span class="string">"response"</span>);</span><br><span class="line">JSONArray data=response.getJSONArray(<span class="string">"data"</span>);</span><br><span class="line">JSONObject info=data.getJSONObject(0);</span><br><span class="line">String province=info.getString(<span class="string">"province"</span>);</span><br><span class="line">String city=info.getString(<span class="string">"city"</span>);</span><br><span class="line">String district=info.getString(<span class="string">"district"</span>);</span><br><span class="line">String address=info.getString(<span class="string">"address"</span>);</span><br><span class="line">System.out.println(province+city+district+address);</span><br></pre></td></tr></table></figure>

          
        
      
    </div>

    <div>
      
    </div>

    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  
  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2016/01/12/php-r-sparkr/" itemprop="url">
                  API configure with Php-r-sparkr
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            <span class="post-meta-item-icon">
              <i class="fa fa-calendar-o"></i>
            </span>
            <span class="post-meta-item-text">Posted on</span>
            <time itemprop="dateCreated" datetime="2016-01-12T17:25:06+08:00" content="2016-01-12">
              2016-01-12
            </time>
          </span>

          
            <span class="post-category" >
              &nbsp; | &nbsp;
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">In</span>
              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/cloud-tech/" itemprop="url" rel="index">
                    <span itemprop="name">cloud-tech</span>
                  </a>
                </span>

                
                

              
            </span>
          

          
            
              <span class="post-comments-count">
                &nbsp; | &nbsp;
                <a href="/2016/01/12/php-r-sparkr/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count disqus-comment-count" data-disqus-identifier="2016/01/12/php-r-sparkr/" itemprop="commentsCount"></span>
                </a>
              </span>
            
          

          

          
          
             <span id="/2016/01/12/php-r-sparkr/" class="leancloud_visitors" data-flag-title="API configure with Php-r-sparkr">
               &nbsp; | &nbsp;
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               <span class="post-meta-item-text">Views </span>
               <span class="leancloud-visitors-count"></span>
              </span>
          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <blockquote>
<hr>
<p>PHP configure</p>
<hr>
</blockquote>
<ul>
<li><p>How to install the nignx on Ubuntu 10.04?<br>Append the appropriate stanza to /etc/apt/sources.list. The Pgp page expalins the signing of th nignx.org released packaging.</p>
<ul>
<li><p>shell script</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">deb http://nginx.org/packages/ubuntu/ lucid nginx</span><br><span class="line">deb-src http://nginx.org/packages/ubuntu/ lucid nginx</span><br></pre></td></tr></table></figure>
</li>
<li><p>then do</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">sudo <span class="_">-s</span></span><br><span class="line">nginx=stable <span class="comment"># use nginx=development for latest development version</span></span><br><span class="line">add-apt-repository ppa:nginx/<span class="variable">$nginx</span></span><br><span class="line">apt-get update</span><br><span class="line">apt-get install nginx</span><br></pre></td></tr></table></figure>
</li>
<li><p>Then finished. Ref. at website<br>(<a href="https://www.nginx.com/resources/wiki/start/topics/tutorials/install/" target="_blank" rel="external">https://www.nginx.com/resources/wiki/start/topics/tutorials/install/</a>)</p>
</li>
</ul>
</li>
</ul>
<ul>
<li><p>PHP Server Nginx</p>
<ul>
<li>$ cd nginx/html</li>
<li>port : 80 –&gt; public port: 8181</li>
<li>example-data.cloudapp.net:8181/index.html</li>
</ul>
</li>
<li><p>Docker</p>
<ul>
<li>$ curl -sSL https//get.docker.com/ | sh</li>
<li>$ sudo docker run hello-world</li>
<li>Add a user with ‘sudo’<ul>
<li>sudo user -aG docker admin</li>
</ul>
</li>
<li>Restart the server<ul>
<li>$ docker run hello-world</li>
</ul>
</li>
</ul>
</li>
</ul>
<blockquote>
<hr>
<p>R installation</p>
<hr>
</blockquote>
<ul>
<li>R Tutorial</li>
<li><p>Update R on Ubuntu</p>
<ul>
<li>$ sudo gedit /etc/apt/sources.list –&gt; open the .list file<ul>
<li>sudo vi /etc/apt/sources.list</li>
<li>#sudo apt-get install gedit</li>
<li><a href="http://cran.r-project.org/becin/linux/ubuntu/" target="_blank" rel="external">http://cran.r-project.org/becin/linux/ubuntu/</a></li>
</ul>
</li>
<li>add the following line into source.list<ul>
<li>deb <a href="http://cran.cnr.berkeley.edu/bin/linux/ubuntu/" target="_blank" rel="external">http://cran.cnr.berkeley.edu/bin/linux/ubuntu/</a> trusty/</li>
<li><h1 id="trusty_for_Ubuntu_14-04"><a href="#trusty_for_Ubuntu_14-04" class="headerlink" title="trusty for Ubuntu 14.04"></a>trusty for Ubuntu 14.04</h1></li>
</ul>
</li>
<li><p>Secure apt, when u see the gpg secure problem, just do the following lines</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="variable">$sudo</span> apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9</span><br><span class="line"><span class="variable">$gpg</span> --keyserver keyserver.ubuntu.com --recv-key E084DAB9</span><br><span class="line"><span class="variable">$gpg</span> <span class="_">-a</span> --export E084DAB9 | sudo apt-key add -</span><br></pre></td></tr></table></figure>
</li>
<li><p>Then, perform this command</p>
<ul>
<li>sudo apt-get update</li>
<li>sudo apt-get install r-base r-base-dev</li>
</ul>
</li>
</ul>
</li>
<li><p>Import library dplyr &amp; FeatureHashing</p>
<ul>
<li>Request R version &gt;3.1.2<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">$ R</span><br><span class="line">&gt; install.packages(<span class="string">"dplyr"</span>)</span><br><span class="line">&gt; library(dplyr)</span><br><span class="line">&gt; install.packages(<span class="string">"FeatureHashing"</span>)</span><br><span class="line">&gt; library(FeatureHashing)</span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
</ul>
<blockquote>
<hr>
<p>R library</p>
<hr>
</blockquote>
<ul>
<li><p>library</p>
<ul>
<li>Matrix</li>
<li>glmnt</li>
<li>FeatureHashing<ul>
<li>Important, the FeatureHashing’s latest version is 0.9, which default parameter transpose is TRUE (but in 0.8 version is FALSE)</li>
</ul>
</li>
<li>glmnet</li>
<li>dplyr</li>
<li>ROC <a href="http://web.expasy.org/pROC/" target="_blank" rel="external">http://web.expasy.org/pROC/</a>  <ul>
<li>install.packages(“pROC”)</li>
<li>library(pROC)</li>
</ul>
</li>
<li>“josnlite” package, parser json file<ul>
<li>A smart json encoder in R <a href="https://www.opencpu.org/posts/jsonlite-a-smarter-json-encoder/" target="_blank" rel="external">https://www.opencpu.org/posts/jsonlite-a-smarter-json-encoder/</a></li>
<li>install.packages(“jsonlite”, repos=”<a href="http://cran.r-project.org" target="_blank" rel="external">http://cran.r-project.org</a>“)</li>
<li>install.packages(“curl”)</li>
</ul>
</li>
<li>“lubridate”, parser date</li>
<li>“json_decode”, function to parser json into array</li>
<li><p>R compare script (* important)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">df = data.frame(A=c(5,6,7,8), B=c(1,7,5,9))</span><br><span class="line">with(df,df[A&gt;B,])</span><br></pre></td></tr></table></figure>
</li>
<li><p>Censored regression in r</p>
<ul>
<li>Each steps <a href="http://stats.stackexchange.com/questions/149091/censored-regression-in-r" target="_blank" rel="external">http://stats.stackexchange.com/questions/149091/censored-regression-in-r</a></li>
<li>Package <a href="https://cran.r-project.org/web/packages/AER/" target="_blank" rel="external">https://cran.r-project.org/web/packages/AER/</a></li>
<li>How to install packages<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">&gt; install.packages(<span class="string">"AER"</span>, lib = <span class="string">"/my/own/R-packages/"</span>)</span><br><span class="line">&gt; library(<span class="string">"AER"</span>, lib.loc=<span class="string">"/my/own/R-packages/"</span>)</span><br><span class="line">Ref http://www.math.usask.ca/~longhai/software/installrpkg.html</span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<blockquote>
<hr>
<p>PHP call R API</p>
<hr>
</blockquote>
<ul>
<li><p>API part</p>
<ul>
<li><p>PHP API</p>
<ul>
<li>API url <a href="http://example-rtb.cloudapp.net:8181/example/api/predict.php" target="_blank" rel="external">http://example-rtb.cloudapp.net:8181/example/api/predict.php</a></li>
<li>PHP calling function methods</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">call_user_func()</span><br><span class="line">call_user_func(<span class="string">'a'</span>, <span class="string">"111"</span>, <span class="string">"222"</span>)</span><br><span class="line">call_user_func_array()</span><br><span class="line">call_user_func_array(<span class="string">'a'</span>, array(<span class="string">"111"</span>, <span class="string">"222"</span>)</span><br></pre></td></tr></table></figure>
</li>
<li><p>RESTful API based OpenCPU</p>
<ul>
<li>example prediction API<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ time curl http://localhost:7509/ocpu/library/exampleApi/R/predict_api/json -H <span class="string">"Content-Type:application/json"</span> <span class="_">-d</span> <span class="string">'&#123;"request":["http://*.cloudapp.net:8181/json/req.txt"]&#125;'</span></span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
<li><p>Install opencpu on Ubuntu cloud server</p>
<ul>
<li>Recommended on Ubuntu 14.04</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#requires ubuntu 14.04 (trusty)</span></span><br><span class="line">sudo add-apt-reporitory -y ppa:opencpu/opencpu-1.5</span><br><span class="line">sudo apt-get update</span><br><span class="line">sudo apt-get upgrade</span><br><span class="line"><span class="comment">#install opencpu server</span></span><br><span class="line">sudo apt-get install -y opencpu</span><br><span class="line"><span class="comment"># optional</span></span><br><span class="line">sudo apt-get isntall -y rstudio-server</span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
<li><p>R command</p>
<ul>
<li><p>Local</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/sparkR --packages com.databricks:spark-csv_2.11:1.2.0</span><br></pre></td></tr></table></figure>
</li>
<li><p>Standalone</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/sparkR --master spark://example-data01:7077 --packages com.databricks:spark-csv_2.11:1.2.0</span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
<li><p>SparkR MLlib</p>
<ul>
<li>An exampel for sparkR MLlib<ul>
<li>Ref <a href="https://github.com/AlbanPhelip/SparkR-example" target="_blank" rel="external">https://github.com/AlbanPhelip/SparkR-example</a></li>
</ul>
</li>
<li>Only the glm() can be used in sparkR<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/sparkR --master spark://example-data01:7077 --packages com.databricks:spark-csv_2.11:1.2.0 /home/admin/Rscript_model_<span class="built_in">test</span>/train_sparkr.R hdfs://masters/Rscript_model/data.txt hdfs://masters/Rscript_model/model.Rds</span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
</ul>
<blockquote>
<hr>
<p>R on SparkR  tutorials for Big Data analysis and Machine Learning as IPython/Jupyter notebooks</p>
<h2 id="https_3A//github-com/jadianes/spark-r-notebooks"><a href="#https_3A//github-com/jadianes/spark-r-notebooks" class="headerlink" title="https://github.com/jadianes/spark-r-notebooks"></a><a href="https://github.com/jadianes/spark-r-notebooks" target="_blank" rel="external">https://github.com/jadianes/spark-r-notebooks</a></h2><hr>
<p>A good example for R app</p>
<hr>
</blockquote>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">http://blog.fens.me/r-app-china-weather/</span><br><span class="line">RandomForest <span class="keyword">in</span> R with parallel trainning.</span><br><span class="line"><span class="comment"># the parallel computing by RandomForest</span></span><br><span class="line">library(randomForest)</span><br><span class="line"> cl &lt;- makeCluster(4)</span><br><span class="line"> registerDoParallel(cl)</span><br><span class="line"> rf &lt;- foreach(ntree=rep(25000, 4),</span><br><span class="line">                  .combine=combine,</span><br><span class="line">                  .packages=<span class="string">'randomForest'</span>) %dopar%</span><br><span class="line">          randomForest(Species~., data=iris, ntree=ntree)</span><br><span class="line">stopCluster(cl)</span><br></pre></td></tr></table></figure>
<blockquote>
<hr>
<p>SparkR configure</p>
<hr>
</blockquote>
<p>SparkR is an R package that provides a light-weight frontend to use Apache Spark from R</p>
<ul>
<li>Ref <a href="https://github.com/amplab-extras/SparkR-pkg" target="_blank" rel="external">https://github.com/amplab-extras/SparkR-pkg</a> | <a href="http://amplab-extras.github.io/SparkR-pkg/" target="_blank" rel="external">http://amplab-extras.github.io/SparkR-pkg/</a></li>
<li><p>Configure the sparkR environment</p>
<ul>
<li>Required<ul>
<li>openjdk 7, R</li>
</ul>
</li>
<li>Steps for set up<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">sudo R</span><br><span class="line">install.packages(<span class="string">"rJava"</span>)</span><br><span class="line">install.packages(<span class="string">"devtools"</span>, dependencies = TRUE)</span><br><span class="line"><span class="comment"># after install rJava &amp; devtools</span></span><br><span class="line">library(devtools)</span><br><span class="line">install_github(<span class="string">"amplab-extras/SparkR-pkg"</span>, subdir=<span class="string">"pkg"</span>)</span><br><span class="line"><span class="comment"># Resolve</span></span><br><span class="line"><span class="comment"># sudo apt-get install r-cran-rjava</span></span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
<li><p>Required</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get install libxml2-dev</span><br><span class="line">sudo apt-get install libcurl4-openssl-dev</span><br><span class="line">sudo apt-get install libcurl4-gnutls-dev</span><br><span class="line">sudo apt-get install curl</span><br><span class="line">/etc/apt/source.list</span><br><span class="line">  <span class="comment">#deb http://cran.rstudio.com/bin/linux/ubuntu trusty/</span></span><br><span class="line">sudo apt-get install libssl-dev</span><br></pre></td></tr></table></figure>
</li>
</ul>
<ul>
<li>Sort the SparkR</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">./bin/sparkR --packages com.databricks:spark-csv_2.11:1.2.0</span><br><span class="line"> sc &lt;- sparkR.init(sparkPackages=<span class="string">"com.databricks:spark-csv_2.11:1.2.0"</span>)</span><br><span class="line">sc &lt;- sparkR.init(master=<span class="string">"spark://example-data01:7077"</span>, sparkEnvir=list(spark.executor.memory=<span class="string">"10g"</span>, spark.cores.max=<span class="string">"4"</span>), sparkPackages=<span class="string">"com.databricks:spark-csv_2.11:1.2.0"</span> )</span><br><span class="line">sqlContext &lt;- sparkRSQL.init(sc)</span><br><span class="line">dataT &lt;- read.df(sqlContext, <span class="string">"hdfs://masters/Rscript_model/data.txt"</span>,<span class="string">"com.databricks.spark.csv"</span>,header=<span class="string">"true"</span>,delimiter=<span class="string">"\t"</span>)</span><br><span class="line">on example-data01: dataT &lt;- read.df(sqlContext, <span class="string">"/home/admin/data/data.txt"</span>,<span class="string">"com.databricks.spark.csv"</span>,header=<span class="string">"true"</span>,delimiter=<span class="string">"\t"</span>, transpose = <span class="string">"true"</span>, is.dgCMatrix = <span class="string">"false"</span>)</span><br><span class="line">Ref script</span><br><span class="line">dataT &lt;- read.df(sqlContext, <span class="string">"/home/admin/data/data.txt"</span>,<span class="string">"com.databricks.spark.csv"</span>,header=<span class="string">"true"</span>,delimiter=<span class="string">"\t"</span>)</span><br><span class="line">head(select(dataT,dataT<span class="variable">$is_win</span>))</span><br><span class="line"><span class="built_in">test</span> &lt;- structure(dataT, package=<span class="string">"SparkR"</span>)</span><br><span class="line">dataT &lt;- read.table(sqlContext, <span class="string">"hdfs://masters/Rscript_model/data.txt"</span>,<span class="string">"com.databricks.spark.csv"</span>,header=<span class="string">"true"</span>,delimiter=<span class="string">"\t"</span>)</span><br><span class="line"><span class="built_in">test</span> &lt;- cbind(select(dataT, <span class="string">"is_win"</span>), select(dataT, <span class="string">"days"</span>), select(dataT, <span class="string">"hours"</span>), select(dataT, <span class="string">"exchange_id"</span>), select(dataT, <span class="string">"app_id"</span>), select(dataT, <span class="string">"publiser_id"</span>), select(dataT, <span class="string">"bidfloor"</span>), select(dataT, <span class="string">"w"</span>), select(dataT, <span class="string">"h"</span>), select(dataT, <span class="string">"os"</span>), select(dataT, <span class="string">"Osv"</span>), select(dataT, <span class="string">"model"</span>), select(dataT, <span class="string">"connectiontype"</span>), select(dataT,<span class="string">"country"</span>), select(dataT, <span class="string">"ua"</span>), select(dataT,<span class="string">"carrier"</span>), select(dataT, <span class="string">"js"</span>), select(dataT, <span class="string">"user"</span>), select(dataT, <span class="string">"carriername"</span>), select(dataT, <span class="string">"app_cat"</span>), select(dataT,<span class="string">"btype"</span>), select(dataT,<span class="string">"mimes"</span>), select(dataT,<span class="string">"badv"</span>), select(dataT,<span class="string">"bcat"</span>))</span><br><span class="line">as.data.frame(<span class="built_in">test</span>)</span><br></pre></td></tr></table></figure>
<blockquote>
<hr>
<p>Output Format using R</p>
<hr>
</blockquote>
<ul>
<li><p>pdf  format</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">pdf(file=<span class="string">"myplot.pdf"</span>)</span><br><span class="line">dev.off()</span><br></pre></td></tr></table></figure>
</li>
<li><p>jpeg format</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">setwd(<span class="string">"path"</span>)</span><br><span class="line">jpeg(file=<span class="string">"myplot.jpeg"</span>)</span><br><span class="line">plot(1:10)</span><br><span class="line">dev.off()</span><br></pre></td></tr></table></figure>
</li>
<li><p>png format</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">png(file=<span class="string">"myplot.png"</span>, <span class="built_in">bg</span>=<span class="string">"transparent"</span>)</span><br><span class="line">dev.off()</span><br><span class="line"><span class="comment">#View the png on Ubuntu, by using "gthumb"</span></span><br><span class="line"><span class="comment">#  sudo apt-get install gthumb</span></span><br><span class="line"><span class="comment">#  $gthumb myplot.png</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>bmp format</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bmp(<span class="string">"myplot.bmp"</span>)</span><br></pre></td></tr></table></figure>
</li>
<li><p>PostScript format</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">postscript(<span class="string">"myplot.ps"</span>)</span><br></pre></td></tr></table></figure>
</li>
<li><p>Windows image file format</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">win.metafile(<span class="string">"myplot.ps"</span>)</span><br></pre></td></tr></table></figure>
</li>
</ul>

          
        
      
    </div>

    <div>
      
    </div>

    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
      

  
  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                
                <a class="post-title-link" href="/2016/01/12/hdfs-spark-2/" itemprop="url">
                  HDFS configure with Zookeeper (02)
                </a>
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            <span class="post-meta-item-icon">
              <i class="fa fa-calendar-o"></i>
            </span>
            <span class="post-meta-item-text">Posted on</span>
            <time itemprop="dateCreated" datetime="2016-01-12T15:15:04+08:00" content="2016-01-12">
              2016-01-12
            </time>
          </span>

          
            <span class="post-category" >
              &nbsp; | &nbsp;
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">In</span>
              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/cloud-tech/" itemprop="url" rel="index">
                    <span itemprop="name">cloud-tech</span>
                  </a>
                </span>

                
                

              
            </span>
          

          
            
              <span class="post-comments-count">
                &nbsp; | &nbsp;
                <a href="/2016/01/12/hdfs-spark-2/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count disqus-comment-count" data-disqus-identifier="2016/01/12/hdfs-spark-2/" itemprop="commentsCount"></span>
                </a>
              </span>
            
          

          

          
          
             <span id="/2016/01/12/hdfs-spark-2/" class="leancloud_visitors" data-flag-title="HDFS configure with Zookeeper (02)">
               &nbsp; | &nbsp;
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               <span class="post-meta-item-text">Views </span>
               <span class="leancloud-visitors-count"></span>
              </span>
          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        
          
            <p>Construct the platform for Big Data project</p>
<h2 id="Configure_Zookeeper"><a href="#Configure_Zookeeper" class="headerlink" title="Configure Zookeeper"></a>Configure Zookeeper</h2><ul>
<li>on each server, configure the java environment<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">$ tar -zxvf zookeeper-3.4.6.tar.gz</span><br><span class="line">$ mv zookeeper-3.4.6 zookeeper</span><br><span class="line">$ <span class="built_in">cd</span> zookeeper/conf</span><br><span class="line">$ cp zoo_sample.cfg zoo.cfg</span><br><span class="line">$ vi zoo.cfg</span><br></pre></td></tr></table></figure>
</li>
</ul>
<table>
<thead>
<tr>
<th>Server role</th>
<th>example-data01（namenode1）</th>
<th>example-data02（namenode2）</th>
<th>example-data03（datanode1）</th>
<th style="text-align:right">example-data04（datanode2）</th>
</tr>
</thead>
<tbody>
<tr>
<td>NameNode</td>
<td>YES</td>
<td>YES</td>
<td>NO</td>
<td style="text-align:right">NO</td>
</tr>
<tr>
<td>DataNode</td>
<td>NO</td>
<td>NO</td>
<td>YES</td>
<td style="text-align:right">YES</td>
</tr>
<tr>
<td>JournalNode</td>
<td>YES</td>
<td>YES</td>
<td>YES</td>
<td style="text-align:right">NO</td>
</tr>
<tr>
<td>ZooKeeper</td>
<td>YES</td>
<td>YES</td>
<td>YES</td>
<td style="text-align:right">NO</td>
</tr>
<tr>
<td>ZKFC</td>
<td>YES</td>
<td>YES</td>
<td>NO</td>
<td style="text-align:right">NO</td>
</tr>
</tbody>
</table>
<ul>
<li>8080 =&gt; example-data01:50070</li>
<li>8000 =&gt; example-data02:50070</li>
</ul>
<h3 id="Hadoop_configure"><a href="#Hadoop_configure" class="headerlink" title="Hadoop configure"></a>Hadoop configure</h3><ul>
<li><p>hosts</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">*.*.*.*  example-data01</span><br><span class="line">*.*.*.*  example-data02</span><br><span class="line">*.*.*.*  example-data03</span><br><span class="line">*.*.*.*  example-data04</span><br></pre></td></tr></table></figure>
</li>
<li><p>$ vi ~/.bashrc</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># vim ~/.bashrc</span></span><br><span class="line"><span class="built_in">export</span> ZOO_HOME=/home/admin/zookeeper</span><br><span class="line"><span class="built_in">export</span> ZOO_LOG_DIR=/home/admin/zookeeper/logs</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$PATH</span>:<span class="variable">$ZOO_HOME</span>/bin</span><br><span class="line"><span class="comment"># source ~/.bashrc</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>$ mkdir -p /data/hadoop/zookeeper/{data,logs}</p>
</li>
<li>$ cp ~/zookeeper/conf/zoo_sample.cfg ~/zookeeper/conf/zoo.cfg</li>
<li><p>$ vim zoo.cfg</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># vim /home/admin/zookeeper/conf/zoo.cfg</span></span><br><span class="line">tickTime=2000</span><br><span class="line">initLimit=10</span><br><span class="line">syncLimit=5</span><br><span class="line">dataDir=/home/admin/zookeeper/data</span><br><span class="line">clientPort=2181</span><br><span class="line">server.1=example-data01:2888:3888</span><br><span class="line">server.2=example-data02:2888:3888</span><br><span class="line">server.3=example-data03:2888:3888</span><br></pre></td></tr></table></figure>
</li>
<li><p>$ create myid in zkData file   -&gt; echo 1 &gt; myid</p>
</li>
<li>$ scp -r zookeeper/* admin@example-data02:~  -&gt; echo 2 &gt; myid</li>
<li>$ scp -r zookeeper/* admin@example-data03:~   -&gt; echo 3 &gt; myid</li>
<li>$ cp ~/zookeeper to each server ( example-data02, example-data03)</li>
<li>on example-data01<ul>
<li>check status by using  </li>
<li>$  ./bin/zkCli.sh -server example-data01:2181</li>
<li>$  or ./bin/zkCli.sh -server example-data02:2181</li>
<li>exit by using “quit”</li>
</ul>
</li>
<li>on each node to perform<ul>
<li>./bin/zkServer.sh start</li>
<li>./bin/zkServer.sh status</li>
</ul>
</li>
</ul>
<h4 id="Configure"><a href="#Configure" class="headerlink" title="Configure"></a>Configure</h4><pre><code>- example-data01(Namenode) example-data02(Sec-namenode) example-data03(datanode)  example-data04 (datanode)
</code></pre><h4 id="On_example-data01"><a href="#On_example-data01" class="headerlink" title="On example-data01"></a>On example-data01</h4><ul>
<li><p>Step 1:</p>
<ul>
<li>vim ~/.bashrc ( then copy the ~/.bashrc to example-data02( namenode) &amp; example-data03 (datanode)</li>
</ul>
</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># java configure</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/lib/jvm/java-8-oracle</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"><span class="built_in">export</span> CLASSPATH=.:<span class="variable">$CLASSPATH</span>:<span class="variable">$JAVA_HOME</span>/lib:<span class="variable">$JAVA_HOME</span>/jre/lib</span><br><span class="line"></span><br><span class="line"><span class="comment"># hadoop configure</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_HOME=/home/admin/hadoop</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$HADOOP_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> HADOOP_CONF_DIR=<span class="variable">$HADOOP_HOME</span>/etc/hadoop</span><br><span class="line"></span><br><span class="line"><span class="comment">#zookeeper configure</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> ZOO_HOME=/home/admin/zookeeper</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$PATH</span>:<span class="variable">$ZOO_HOME</span>/bin</span><br><span class="line"><span class="built_in">export</span> ZOO_LOG_DIR=/home/admin/zookeeper/logs</span><br><span class="line"></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_COMMON_LIB_NATIVE_DIR=<span class="variable">$HADOOP_HOME</span>/lib/native</span><br><span class="line"><span class="built_in">export</span> HADOOP_OPTS=<span class="string">"-Djava.library.path=<span class="variable">$HADOOP_HOME</span>/lib/native"</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_PID_DIR=/home/admin/hadoop/pids</span><br><span class="line"><span class="built_in">export</span> HADOOP_COMMON_HOME=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_HDFS_HOME=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$HADOOP_HOME</span>/sbin:<span class="variable">$PATH</span></span><br></pre></td></tr></table></figure>
<ul>
<li>Step 2:</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">$ mkdir -p /home/admin/hadoop/&#123;pids,storage&#125;</span><br><span class="line">$ mkdir -p /home/admin/hadoop/storage/&#123;hdfs,tmp,journal&#125;</span><br><span class="line">$ mkdir -p /home/admin/hadoop/storage/hdfs/&#123;name,data&#125;</span><br><span class="line"><span class="keyword">in</span> example-data03 &amp; example-data04</span><br><span class="line">$ mkdir -p /datadriver/hdfs/data</span><br><span class="line">$ sudo chown admin /datadriver/hdfs/data/</span><br></pre></td></tr></table></figure>
<ul>
<li>Step 3:</li>
</ul>
<blockquote>
<p>configure the core-site.xml, hadoop-env.sh, hdfs-site.xml</p>
</blockquote>
<ul>
<li>configure core-site.xml</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;fs.defaultFS&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;hdfs://masters&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;file:/home/admin/hadoop/storage/tmp&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;ha.zookeeper.quorum&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;example-data01:2181,example-data02:2181,example-data03:2181&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;hadoop.native.lib&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line">configure  hdfs-site.xml</span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.nameservices&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;masters&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.ha.namenodes.masters&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;nn1,nn2&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.name.dir&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;file:///home/admin/hadoop/storage/hdfs/name&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.datanode.data.dir&lt;/name&gt;</span><br><span class="line">//                &lt;value&gt;file:/home/admin/hadoop/storage/hdfs/data&lt;/value&gt;</span><br><span class="line">                &lt;value&gt;file:///datadriver/hdfs/data&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;2&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.webhdfs.enabled&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.rpc-address.masters.nn1&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;example-data01:9000&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.http-address.masters.nn1&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;example-data01:50070&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.rpc-address.masters.nn2&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;example-data02:9000&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.http-address.masters.nn2&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;example-data02:50070&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.shared.edits.dir&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;qjournal://example-data01:8485;example-data02:8485;example-data03:8485/masters&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.journalnode.edits.dir&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;/home/admin/hadoop/storage/journal&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.ha.fencing.methods&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;sshfence(hdfs)</span><br><span class="line">                shell(/bin/<span class="literal">true</span>)&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.ha.fencing.ssh.private-key-files&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;/home/admin/.ssh/id_rsa&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.ha.fencing.ssh.connect-timeout&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;30000&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.ha.automatic-failover.enabled&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.client.failover.proxy.provider.masters&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
<ul>
<li>configure hadoop-env.sh</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/lib/jvm/java-8-oracle</span><br></pre></td></tr></table></figure>
<ul>
<li>configure slaves</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">example-data03</span><br><span class="line">example-data04</span><br></pre></td></tr></table></figure>
<ul>
<li>Step 4:</li>
</ul>
<blockquote>
<p>scp hadoop to each node ( example-data02, example-data03, example-data04)</p>
</blockquote>
<ul>
<li>Step 5:</li>
</ul>
<blockquote>
<p>Setup the  HDFS Cluster with ZooKeeper Using Demo (example)</p>
</blockquote>
<ul>
<li><p>on the namenode1 (example-data01)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ hdfs zkfc -formatZK</span><br></pre></td></tr></table></figure>
</li>
<li><p>on each zookeeper node</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">$ hadoop-daemon.sh start journalnode</span><br><span class="line">// $ ./sbin/hadoop-daemons.sh --hostnames <span class="string">'example-data01 example-data02 example-data03'</span> start journalnode  // this <span class="built_in">command</span> is used to start the journalnode of zookeeper</span><br></pre></td></tr></table></figure>
</li>
<li><p>on the namenode1 (example-data01)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ hdfs namenode -format</span><br></pre></td></tr></table></figure>
</li>
<li><p>on the namenode1 (example-data01)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ ./sbin/hadoop-daemon.sh start namenode</span><br></pre></td></tr></table></figure>
</li>
<li><p>on namenode2 (example-data02)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ hdfs namenode -bootstrapStandby</span><br></pre></td></tr></table></figure>
</li>
<li><p>on namenode2 (example-data02)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ ./sbin/hadoop-daemon.sh start namenode</span><br></pre></td></tr></table></figure>
</li>
<li><p>on the two namenode (namenode1, namenode2)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ ./sbin/hadoop-daemon.sh start zkfc</span><br></pre></td></tr></table></figure>
</li>
<li><p>on the all datanode (datanode1, datanode2)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ ./sbin/hadoop-daemon.sh start datanode</span><br></pre></td></tr></table></figure>
</li>
<li><p>still now, we can see the following name on namenode1 (example-data01)</p>
<blockquote>
<p>QuorumPeerMain, NameNode, DFSZKFailoverController(basically), JournalNode</p>
</blockquote>
</li>
</ul>
<ul>
<li><p>Step 6: Testing the HDFS’s  function<br>-</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#  login on web</span></span><br><span class="line">example-data01: namenode (active)  =&gt; example-data01:9000</span><br><span class="line">website: example-hadoop.cloudapp.net:8080</span><br><span class="line">example-data02: namenode (standby)  =&gt; example-data02:9000</span><br><span class="line">website: example-hadoop.cloudapp.net:8000</span><br><span class="line"><span class="comment"># $ hadoop-daemon.sh start namenode</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>Step 7: Import , excute on namenode1 (example-data01)</p>
<ul>
<li><p>Before Starting HDFS , Formating the zookeeper</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ hdfs zkfc -formatZK</span><br><span class="line">Start the HDFS  --&gt; $ <span class="built_in">cd</span> /home/admin/hadoop &amp;&amp; sbin/start-dfs.sh</span><br><span class="line">Stop the HDFS  --&gt; $ <span class="built_in">cd</span> /home/admin/hadoop &amp;&amp; sbin/stop-dfs.sh</span><br><span class="line">$ start-dfs.sh</span><br></pre></td></tr></table></figure>
</li>
<li><p>check the status on two namenode</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">$ hdfs haadmin -getServiceState nn1 --&gt; active</span><br><span class="line">$ hdfs haadmin -getServiceState nn2 --&gt; standby</span><br></pre></td></tr></table></figure>
</li>
<li><p>when the stop-dfs.sh turn to <a href="http://taoo.iteye.com/blog/1730406" target="_blank" rel="external">error</a></p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">$ vi ~/.bashrc</span><br><span class="line"><span class="built_in">export</span> HADOOP_PID_DIR=/home/hadoop/pids</span><br></pre></td></tr></table></figure>
</li>
</ul>
</li>
<li><p>hdfs dfsadmin -report</p>
</li>
</ul>
<h3 id="Spark_on_Yarn_Reference"><a href="#Spark_on_Yarn_Reference" class="headerlink" title="Spark on Yarn Reference"></a><a href="http://wuchong.me/blog/2015/04/04/spark-on-yarn-cluster-deploy/" target="_blank" rel="external">Spark on Yarn Reference</a></h3><p>configure additional two file<br>yarn-site.xml &amp; maperd-site.xml</p>

          
        
      
    </div>

    <div>
      
    </div>

    <footer class="post-footer">
      

      

      
      
        <div class="post-eof"></div>
      
    </footer>
  </article>


    
  </section>

  
  <nav class="pagination">
    <a class="extend prev" rel="prev" href="/page/3/"><i class="fa fa-angle-left"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/3/">3</a><span class="page-number current">4</span><a class="page-number" href="/page/5/">5</a><a class="extend next" rel="next" href="/page/5/"><i class="fa fa-angle-right"></i></a>
  </nav>



          </div>
          


          

        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      

      <section class="site-overview sidebar-panel  sidebar-panel-active ">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/images/avatar.png"
               alt="SZhou" />
          <p class="site-author-name" itemprop="name">SZhou</p>
          <p class="site-description motion-element" itemprop="description">The unexamined life is not worth living. --Socrates</p>
        </div>
        <nav class="site-state motion-element">
          <div class="site-state-item site-state-posts">
            <a href="/archives">
              <span class="site-state-item-count">24</span>
              <span class="site-state-item-name">posts</span>
            </a>
          </div>
          
          
            <div class="site-state-item site-state-categories">
              
                <span class="site-state-item-count">5</span>
                <span class="site-state-item-name">categories</span>
              
            </div>
          

          
            <div class="site-state-item site-state-tags">
              
                <span class="site-state-item-count">22</span>
                <span class="site-state-item-name">tags</span>
              
            </div>
          

        </nav>

        
          <div class="feed-link motion-element">
            <a href="/atom.xml" rel="alternate">
              <i class="fa fa-rss"></i>
              RSS
            </a>
          </div>
        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/snakecy" target="_blank">
                  
                    <i class="fa fa-github"></i> GitHub
                  
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="https://hk.linkedin.com/in/snakecy" target="_blank">
                  
                    <i class="fa fa-linkedin"></i> LinkedIn
                  
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://weibo.com/snakecys" target="_blank">
                  
                    <i class="fa fa-weibo"></i> Weibo
                  
                </a>
              </span>
            
          
        </div>

        
        
          <div class="cc-license motion-element" itemprop="license">
            <a href="http://creativecommons.org/licenses/by-nc-sa/4.0" class="cc-opacity" target="_blank">
              <img src="/images/cc-by-nc-sa.svg" alt="Creative Commons" />
            </a>
          </div>
        

        <div class="links-of-author motion-element">
          
            <p class="site-author-name">Links</p>
            
              <span class="links-of-author-item">
                <a href="http://snakecy.github.io/topics" target="_blank">DataTopics</a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://www.chinabyte.com/" target="_blank">Chinabyte</a>
              </span>
            
          
        </div>

      </section>

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <script async src="//dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js"></script>
<div class="copyright" >
  
  &copy; 
  <span itemprop="copyrightYear">2016</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">SZhou</span>
</div>

<div class="powered-by">
  Powered by <a class="theme-link" href="http://hexo.io">Hexo</a>
</div>

<div class="theme-info">
  Theme -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Mist
  </a>
</div>

<span id="busuanzi_container_site_pv">
  &nbsp; | &nbsp; <span id="busuanzi_value_site_pv"></span> hits from
</span>

<span id="busuanzi_container_site_uv"> <span id="busuanzi_value_site_uv"></span> vistors
</span>



      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
    </div>
  </div>

  


  




<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>


  <script type="text/javascript" src="/vendors/jquery/index.js?v=2.1.3"></script>

  <script type="text/javascript" src="/vendors/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  <script type="text/javascript" src="/vendors/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  <script type="text/javascript" src="/vendors/velocity/velocity.min.js"></script>

  <script type="text/javascript" src="/vendors/velocity/velocity.ui.min.js"></script>

  <script type="text/javascript" src="/vendors/fancybox/source/jquery.fancybox.pack.js"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=0.5.0"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=0.5.0"></script>



  
  

  

  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=0.5.0"></script>



  



  

    <script type="text/javascript">
      var disqus_shortname = 'snakecy';
      var disqus_identifier = 'page/4/index.html';
      var disqus_title = '';
      var disqus_url = '';

      function run_disqus_script(disqus_script){
        var dsq = document.createElement('script');
        dsq.type = 'text/javascript';
        dsq.async = true;
        dsq.src = '//' + disqus_shortname + '.disqus.com/' + disqus_script;
        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
      }

      run_disqus_script('count.js');
      
    </script>
  



  
  
  
  <script type="text/x-mathjax-config">
    MathJax.Hub.Config({
      tex2jax: {
        inlineMath: [ ['$','$'], ["\\(","\\)"]  ],
        processEscapes: true,
        skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
      }
    });
  </script>

  <script type="text/x-mathjax-config">
    MathJax.Hub.Queue(function() {
      var all = MathJax.Hub.getAllJax(), i;
      for (i=0; i < all.length; i += 1) {
        all[i].SourceElement().parentNode.className += ' has-jax';
      }
    });
  </script>

  
    <script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
  


  

  
  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.1.js"></script>
  <script>AV.initialize("ICX7w1eUjsdhsI9peY8NhDy8-gzGzoHsz", "LNeLg9XFL6IJ6srOgJ48UiHT");</script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");

      $visitors.each(function () {
        entries.push( $(this).attr("id").trim() );
      });

      query.containedIn('url', entries);
      query.find()
        .done(function (results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';

          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }

          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);

            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
        })
        .fail(function (object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);

      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            counter.save(null, {
              success: function(counter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(counter.get('time'));
              },
              error: function(counter, error) {
                console.log('Failed to save Visitor num, with error message: ' + error.message);
              }
            });
          } else {
            var newcounter = new Counter();
            newcounter.set("title", title);
            newcounter.set("url", url);
            newcounter.set("time", 1);
            newcounter.save(null, {
              success: function(newcounter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(newcounter.get('time'));
              },
              error: function(newcounter, error) {
                console.log('Failed to create');
              }
            });
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }

    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });
  </script>




</body>
</html>
